#!/usr/bin/env bash
export PYSPARK_DRIVER_PYTHON=/home/hdfs/Python/bin/python3
export PYSPARK_PYTHON=/home/hdfs/Python/bin/python3
export SPARK_WORKER_INSTANCES=2
export CORES_PER_WORKER=1 
export TOTAL_CORES=$((${CORES_PER_WORKER}*${SPARK_WORKER_INSTANCES}))
export HADOOP_HDFS_HOME=/home/hdfs/bigdata/hadoop-2.7.4
${SPARK_HOME}/bin/spark-submit \
--master spark://bigdata.lkl.com:7077 \
${TFoS_HOME}/examples/mnist/mnist_data_setup.py \
--output ${TFoS_HOME}/examples/mnist/csv \
--format csv

